package com.lheia.process.guangdong.guangzhou.panyu

import com.lheia.jcb.spider.ApprovalPublicity
import com.lheia.jcb.spider.IpeSpiderData
import com.lheia.process.BaseProcess
import com.lheia.tool.ToolHttpConnect
import com.lheia.tool.ToolString
import com.lheia.tool.ToolUserAgents
import com.lheia.util.PageFieldConstants
import com.lheia.util.RequestExtraConstants
import com.lheia.util.ToolUrl
import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Request
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.utils.HttpConstant

/**
 * 审批
 */
class SPProcess implements BaseProcess {
    private Site site = Site.me().setCycleRetryTimes(3).setUserAgent(ToolUserAgents.userAgentRandom())
    private String province = "广东省"
    private String city = "广州市"
    private String county = "番禺区"
    private String approveDept = "番禺区人民政府"


    private pageQueryData = []

    void setIfTest(boolean ifTest) {
        this.ifTest = ifTest
    }
    protected boolean ifTest = true
    /**
     * process the page, extract urls to fetch, extract the data and store
     *
     * @param page page
     */
    @Override
    /**
     * 主要思路使用xpath解析列表也对象添加到请求队列中
     *
     */
    void process(Page page) {
        def methodName = page.getRequest().getExtra(RequestExtraConstants.CALLBACK)
        if (methodName) {
            this."$methodName"(page)
        }
    }

    @Override
    def getBeginRequest(String url) {
        List<Request> list = new ArrayList<>()
        Request request = new Request(url)
        /**
         * 使用无头浏览器
         */
        request.putExtra(RequestExtraConstants.PHANTOM_JS, true)
        /**
         * 不计入去重
         */
        request.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
        request.setMethod(HttpConstant.Method.GET);
        /**
         * 处理回调
         */
        request.putExtra(RequestExtraConstants.CALLBACK, "processList")
        list << request
        return list
    }


/***
 * 列表处理
 * @param page
 */
    void processList(Page page) {
        /**列表node查找***/
        def trList = page.getHtml().getDocument().select("div.news_list").select("li")
        trList.each {
            def listNode = it.getElementsByTag("a").get(0)
            def detailUrl = listNode.attr("href")
            def title = listNode.attr("title")
            if(title.contains("环保验收") || title.contains("风险评估")){
                /***
                 * 过滤title是否抓取抓取之后
                 */
                def abs = ToolUrl.getAbsoluteURL(page.getUrl().toString(), detailUrl)
                def detailRequest = new Request(abs)
                println String.join(",", [title, abs])
                detailRequest.putExtra(RequestExtraConstants.CALLBACK, "processPagingSP")
                page.addTargetRequest(detailRequest)
            }else{
                println("过滤："+title)
            }
        }
        page.getResultItems().setSkip(true)
        processPaging(page)
    }
/**
 * 分页处理
 * @param page
 */
    void processPaging(Page page) {
        def doc = page.getHtml().getDocument()
        println page
        /***
         * 获取总页数
         */
        def nextPageUrl = doc.select("a:containsOwn(下一页)")?.get(0)?.attr("href")
        if (nextPageUrl) {
            /***
             * 分页请求不需要进行去重
             */
            pageRequest.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
            /**
             * 是否使用phantomJS
             */
            pageRequest.putExtra(RequestExtraConstants.PHANTOM_JS, page.getRequest().getExtra(RequestExtraConstants.PHANTOM_JS))
            /**
             * 处理回调
             */
            pageRequest.putExtra(RequestExtraConstants.CALLBACK, "processList")
            println("下一页," + pageRequest)
            page.addTargetRequest(pageRequest)
        }
    }

    void processPagingSP(Page page) {
        def doc = page.getHtml().getDocument()
        println(doc)
        def approval = new ApprovalPublicity()
//        ipeSpiderData.companyName = doc.select("td.xzcf_xx").get(1).text()
//        ipeSpiderData.messageSource = doc.select("td.xzcf_xx").get(0).text()
//        ipeSpiderData.pubOrg = doc.select("td.xzcf_xx").get(2).text()
//        ipeSpiderData.province = this.province
//        ipeSpiderData.city = this.city
//        ipeSpiderData.county = this.county
//        ipeSpiderData.illegalDetail = doc.select("td.xzcf_jds").text()
//        ipeSpiderData.status = "处罚"
//        ipeSpiderData.timeOccurrence = com.lheia.tool.ToolDateTime.extractDate(doc.select("td.xzcf_xx").get(3).text())
//        page.putField(PageFieldConstants.RESULT_DATA, ipeSpiderData)
    }


/**
 * 如果为测试环境或者开发环境不计入redis去重只使用redisList队列
 * @param page
 */
    void processTargetResquest(Page page) {
        if (ifTest) {
            println(page)
            page.getTargetRequests().each {
                it.putExtra(RequestExtraConstants.NO_NEED_TO_REMOVE, true)
            }
        }
    }


/**
 * get the site settings
 *
 * @return site* @see Site
 */
    @Override
    Site getSite() {
        return site
    }

}
